{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "ade0633c",
   "metadata": {},
   "source": [
    "QLearning使用表格估计Q函数,不便于扩展.\n",
    "\n",
    "所以使用神经网络估计Q函数."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "91162df8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAASuklEQVR4nO3dfUxTZ98H8C8VWuXllKGjHZFGE90c8W1DxaN5ni3aG6bEzMmSbTGOGaPRFSOymIzEl+i2YFwynZviP5u6LM6FJW6R+DJSFbNYRXE8QVTueccNorad+nAKKC3Q6/lj4TyroqNQelH9fpKTeM71a/s7l/ab9lwcjBNCCBARSWCQ3QARPb0YQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJI20ANq1axfGjBmD4cOHIycnBzU1NbJaISJJpATQ999/j5KSEmzatAkXL17ElClTkJeXB6/XK6MdIpIkTsbNqDk5OZg+fTq+/PJLAEAwGERmZiZWr16NDz/8MNrtEJEk8dF+wUAggNraWpSWlurHDAYD7HY7XC5Xr4/x+/3w+/36fjAYxN27dzFy5EjExcUNes9EFB4hBFpbW5GRkQGD4dFftKIeQLdv30Z3dzcsFkvIcYvFgqtXr/b6mLKyMmzevDka7RFRBDU3N2P06NGPHI96APVHaWkpSkpK9H1N02Cz2dDc3AxFUSR2RkS98fl8yMzMREpKymProh5Ao0aNwrBhw+DxeEKOezweWK3WXh9jMplgMpkeOq4oCgOIaAj7p0skUV8FMxqNyM7OhtPp1I8Fg0E4nU6oqhrtdohIIilfwUpKSlBYWIhp06ZhxowZ2LFjB9rb27F06VIZ7RCRJFIC6K233sKff/6JjRs3wu12Y+rUqTh27NhDF6aJ6Mkm5eeABsrn88FsNkPTNF4DIhqC+voe5b1gRCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikibsADp9+jQWLFiAjIwMxMXF4ccffwwZF0Jg48aNeO655zBixAjY7Xb89ttvITV3797F4sWLoSgKUlNTsWzZMrS1tQ3oRIgo9oQdQO3t7ZgyZQp27drV6/i2bduwc+dO7NmzB+fOnUNSUhLy8vLQ0dGh1yxevBgNDQ2oqqpCZWUlTp8+jRUrVvT/LIgoNokBACAOHTqk7weDQWG1WsWnn36qH2tpaREmk0l89913QgghLl++LACI8+fP6zVHjx4VcXFx4saNG316XU3TBAChadpA2ieiQdLX92hErwFdv34dbrcbdrtdP2Y2m5GTkwOXywUAcLlcSE1NxbRp0/Qau90Og8GAc+fO9fq8fr8fPp8vZCOi2BfRAHK73QAAi8USctxisehjbrcb6enpIePx8fFIS0vTax5UVlYGs9msb5mZmZFsm4gkiYlVsNLSUmiapm/Nzc2yWyKiCIhoAFmtVgCAx+MJOe7xePQxq9UKr9cbMt7V1YW7d+/qNQ8ymUxQFCVkI6LYF9EAGjt2LKxWK5xOp37M5/Ph3LlzUFUVAKCqKlpaWlBbW6vXnDhxAsFgEDk5OZFsh4iGuPhwH9DW1oZr167p+9evX0ddXR3S0tJgs9lQXFyMjz/+GOPHj8fYsWOxYcMGZGRkYOHChQCAF198Ea+99hqWL1+OPXv2oLOzE0VFRXj77beRkZERsRMjohgQ7vLayZMnBYCHtsLCQiHEX0vxGzZsEBaLRZhMJjF37lzR2NgY8hx37twR77zzjkhOThaKooilS5eK1tbWiC/xEZEcfX2PxgkhhMT86xefzwez2QxN03g9iGgI6ut7NCZWwYjoycQAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKQJ+7/lIYq01lu/IdB2V983mdORnD5WYkcULQwgks5T78T/Xr+o7z/74n8xgJ4S/ApGRNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpAkrgMrKyjB9+nSkpKQgPT0dCxcuRGNjY0hNR0cHHA4HRo4cieTkZBQUFMDj8YTUNDU1IT8/H4mJiUhPT8e6devQ1dU18LMhopgSVgBVV1fD4XDg7NmzqKqqQmdnJ3Jzc9He3q7XrF27FocPH0ZFRQWqq6tx8+ZNLFq0SB/v7u5Gfn4+AoEAzpw5g/3792Pfvn3YuHFj5M6KiGKDGACv1ysAiOrqaiGEEC0tLSIhIUFUVFToNVeuXBEAhMvlEkIIceTIEWEwGITb7dZrysvLhaIowu/39+l1NU0TAISmaQNpn4aI346Xi5o9y/XtevU3sluiAerre3RA14A0TQMApKWlAQBqa2vR2dkJu92u10yYMAE2mw0ulwsA4HK5MGnSJFgsFr0mLy8PPp8PDQ0Nvb6O3++Hz+cL2Ygo9vU7gILBIIqLizF79mxMnDgRAOB2u2E0GpGamhpSa7FY4Ha79Zq/h0/PeM9Yb8rKymA2m/UtMzOzv20T0RDS7wByOBy4dOkSDh48GMl+elVaWgpN0/Stubl50F+TiAZfv25GLSoqQmVlJU6fPo3Ro0frx61WKwKBAFpaWkI+BXk8HlitVr2mpqYm5Pl6Vsl6ah5kMplgMpn60yoRDWFhfQISQqCoqAiHDh3CiRMnMHZs6B3L2dnZSEhIgNPp1I81NjaiqakJqqoCAFRVRX19Pbxer15TVVUFRVGQlZU1kHMhohgT1icgh8OBAwcO4KeffkJKSop+zcZsNmPEiBEwm81YtmwZSkpKkJaWBkVRsHr1aqiqipkzZwIAcnNzkZWVhSVLlmDbtm1wu91Yv349HA4HP+UQPWXCCqDy8nIAwKuvvhpyfO/evXjvvfcAANu3b4fBYEBBQQH8fj/y8vKwe/duvXbYsGGorKzEqlWroKoqkpKSUFhYiC1btgzsTIgo5sQJIYTsJsLl8/lgNpuhaRoURZHdDg3QtZ/3PPQLycb89xKJHdFA9fU9ynvBiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0oT1P6MS9Yff78f9+/cfOd7Z2flAfQAtLS2PrE9MTITRaIxUeyQRA4gG3TfffIPNmzc/cnzdmy9BnfCcvv/DDxUoX7rpkfU7duzAm2++GdEeSQ4GEA26trY23Lhx45Hj7fcm4dr9l3G3MwPPJvyB1rZ/P7b+3r17g9EmSRDWNaDy8nJMnjwZiqJAURSoqoqjR4/q4x0dHXA4HBg5ciSSk5NRUFAAj8cT8hxNTU3Iz89HYmIi0tPTsW7dOnR1dUXmbCgm/ef+VFy791cA/fteDv7oyJLdEkVJWAE0evRobN26FbW1tbhw4QLmzJmD119/HQ0NDQCAtWvX4vDhw6ioqEB1dTVu3ryJRYsW6Y/v7u5Gfn4+AoEAzpw5g/3792Pfvn3YuHFjZM+KYkpbVyp6/ikKGNDenSqzHYqisAJowYIFmD9/PsaPH4/nn38en3zyCZKTk3H27FlomoavvvoKn332GebMmYPs7Gzs3bsXZ86cwdmzZwEAP//8My5fvoxvv/0WU6dOxbx58/DRRx9h165dCAQCg3KCNPRlmP6D+Dg/AAFj3H1Yjddlt0RR0u9rQN3d3aioqEB7eztUVUVtbS06Oztht9v1mgkTJsBms8HlcmHmzJlwuVyYNGkSLBaLXpOXl4dVq1ahoaEBL730Ulg9XL16FcnJyf09BYqSB7+GP+jX/6mCufkPtHSlIy3BDfetq4+tv3nzJi5fvhzJFinC2tra+lQXdgDV19dDVVV0dHQgOTkZhw4dQlZWFurq6mA0GpGamhpSb7FY4Ha7AQButzskfHrGe8Yexe/3w+/36/s+nw8AoGkarx/FgMctwQNAdd3vAH7v8/Pdu3fvscv0JF97e3uf6sIOoBdeeAF1dXXQNA0//PADCgsLUV1dHXaD4SgrK+t1GTcnJweKogzqa9PAnTt3LqLPN27cOMyaNSuiz0mR1fMh4Z+E/ZPQRqMR48aNQ3Z2NsrKyjBlyhR8/vnnsFqtCAQe/gEyj8cDq9UKALBarQ99HO/Z76npTWlpKTRN07fm5uZw2yaiIWjAt2IEg0H4/X5kZ2cjISEBTqdTH2tsbERTUxNUVQUAqKqK+vp6eL1evaaqqgqKoiAr69FLryaTSV/679mIKPaF9RWstLQU8+bNg81mQ2trKw4cOIBTp07h+PHjMJvNWLZsGUpKSpCWlgZFUbB69WqoqoqZM2cCAHJzc5GVlYUlS5Zg27ZtcLvdWL9+PRwOB0wm06CcIBENXWEFkNfrxbvvvotbt27BbDZj8uTJOH78OP71r38BALZv3w6DwYCCggL4/X7k5eVh9+7d+uOHDRuGyspKrFq1CqqqIikpCYWFhdiyZUtkz4qGlJ5PsJHC+8CeHHFCCCG7iXD5fD6YzWZomsavYzGgtbU1oqtWaWlpSEpKitjzUeT19T3Ke8Fo0KWkpCAlJUV2GzQE8fcBEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImniZTfQH0IIAIDP55PcCRH1pue92fNefZSYDKA7d+4AADIzMyV3QkSP09raCrPZ/MjxmAygtLQ0AEBTU9NjT45C+Xw+ZGZmorm5GYqiyG4nJnDO+kcIgdbWVmRkZDy2LiYDyGD469KV2WzmP4p+UBSF8xYmzln4+vLhgBehiUgaBhARSROTAWQymbBp0yaYTCbZrcQUzlv4OGeDK0780zoZEdEgiclPQET0ZGAAEZE0DCAikoYBRETSxGQA7dq1C2PGjMHw4cORk5ODmpoa2S1JU1ZWhunTpyMlJQXp6elYuHAhGhsbQ2o6OjrgcDgwcuRIJCcno6CgAB6PJ6SmqakJ+fn5SExMRHp6OtatW4eurq5onoo0W7duRVxcHIqLi/VjnLMoETHm4MGDwmg0iq+//lo0NDSI5cuXi9TUVOHxeGS3JkVeXp7Yu3evuHTpkqirqxPz588XNptNtLW16TUrV64UmZmZwul0igsXLoiZM2eKWbNm6eNdXV1i4sSJwm63i19//VUcOXJEjBo1SpSWlso4paiqqakRY8aMEZMnTxZr1qzRj3POoiPmAmjGjBnC4XDo+93d3SIjI0OUlZVJ7Gro8Hq9AoCorq4WQgjR0tIiEhISREVFhV5z5coVAUC4XC4hhBBHjhwRBoNBuN1uvaa8vFwoiiL8fn90TyCKWltbxfjx40VVVZV45ZVX9ADinEVPTH0FCwQCqK2thd1u148ZDAbY7Xa4XC6JnQ0dmqYB+P8bdmtra9HZ2RkyZxMmTIDNZtPnzOVyYdKkSbBYLHpNXl4efD4fGhoaoth9dDkcDuTn54fMDcA5i6aYuhn19u3b6O7uDvlLBwCLxYKrV69K6mroCAaDKC4uxuzZszFx4kQAgNvthtFoRGpqakitxWKB2+3Wa3qb056xJ9HBgwdx8eJFnD9//qExzln0xFQA0eM5HA5cunQJv/zyi+xWhrTm5masWbMGVVVVGD58uOx2nmox9RVs1KhRGDZs2EOrER6PB1arVVJXQ0NRUREqKytx8uRJjB49Wj9utVoRCATQ0tISUv/3ObNarb3Oac/Yk6a2thZerxcvv/wy4uPjER8fj+rqauzcuRPx8fGwWCycsyiJqQAyGo3Izs6G0+nUjwWDQTidTqiqKrEzeYQQKCoqwqFDh3DixAmMHTs2ZDw7OxsJCQkhc9bY2IimpiZ9zlRVRX19Pbxer15TVVUFRVGQlZUVnROJorlz56K+vh51dXX6Nm3aNCxevFj/M+csSmRfBQ/XwYMHhclkEvv27ROXL18WK1asEKmpqSGrEU+TVatWCbPZLE6dOiVu3bqlb/fu3dNrVq5cKWw2mzhx4oS4cOGCUFVVqKqqj/csKefm5oq6ujpx7Ngx8eyzzz5VS8p/XwUTgnMWLTEXQEII8cUXXwibzSaMRqOYMWOGOHv2rOyWpAHQ67Z371695v79++L9998XzzzzjEhMTBRvvPGGuHXrVsjz/P7772LevHlixIgRYtSoUeKDDz4QnZ2dUT4beR4MIM5ZdPDXcRCRNDF1DYiIniwMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISJr/AxFJMCBRCGiGAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('CartPole-v1', render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, terminated, truncated, info = self.env.step(action)\n",
    "        over = terminated or truncated\n",
    "\n",
    "        #限制最大步数\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 200:\n",
    "            over = True\n",
    "        \n",
    "        #没坚持到最后,扣分\n",
    "        if over and self.step_n < 200:\n",
    "            reward = -1000\n",
    "\n",
    "        return state, reward, over\n",
    "\n",
    "    #打印游戏图像\n",
    "    def show(self):\n",
    "        from matplotlib import pyplot as plt\n",
    "        plt.figure(figsize=(3, 3))\n",
    "        plt.imshow(self.env.render())\n",
    "        plt.show()\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()\n",
    "\n",
    "env.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "ecfbe912",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Sequential(\n",
       "  (0): Linear(in_features=4, out_features=64, bias=True)\n",
       "  (1): ReLU()\n",
       "  (2): Linear(in_features=64, out_features=64, bias=True)\n",
       "  (3): ReLU()\n",
       "  (4): Linear(in_features=64, out_features=2, bias=True)\n",
       ")"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import torch\n",
    "\n",
    "#定义模型,评估状态下每个动作的价值\n",
    "model = torch.nn.Sequential(\n",
    "    torch.nn.Linear(4, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 64),\n",
    "    torch.nn.ReLU(),\n",
    "    torch.nn.Linear(64, 2),\n",
    ")\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "84cbf0ff",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "d:\\appDir\\python3.10\\lib\\site-packages\\gym\\utils\\passive_env_checker.py:233: DeprecationWarning: `np.bool8` is a deprecated alias for `np.bool_`.  (Deprecated NumPy 1.24)\n",
      "  if not isinstance(terminated, (bool, np.bool8)):\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "-979.0"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "#玩一局游戏并记录数据\n",
    "def play(show=False):\n",
    "    data = []\n",
    "    reward_sum = 0\n",
    "\n",
    "    state = env.reset()\n",
    "    over = False\n",
    "    while not over:\n",
    "        action = model(torch.FloatTensor(state).reshape(1, 4)).argmax().item()\n",
    "        if random.random() < 0.1:\n",
    "            action = env.action_space.sample()\n",
    "\n",
    "        next_state, reward, over = env.step(action)\n",
    "\n",
    "        data.append((state, action, reward, next_state, over))\n",
    "        reward_sum += reward\n",
    "\n",
    "        state = next_state\n",
    "\n",
    "        if show:\n",
    "            display.clear_output(wait=True)\n",
    "            env.show()\n",
    "\n",
    "    return data, reward_sum\n",
    "\n",
    "\n",
    "play()[-1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "95ce610e",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\Administrator\\AppData\\Local\\Temp\\ipykernel_9152\\2282743005.py:27: UserWarning: Creating a tensor from a list of numpy.ndarrays is extremely slow. Please consider converting the list to a single numpy.ndarray with numpy.array() before converting to a tensor. (Triggered internally at ..\\torch\\csrc\\utils\\tensor_new.cpp:248.)\n",
      "  state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(221,\n",
       " (array([-0.03021992,  0.01805801,  0.01001038, -0.02743726], dtype=float32),\n",
       "  1,\n",
       "  1.0,\n",
       "  array([-0.02985876,  0.21303499,  0.00946163, -0.31694508], dtype=float32),\n",
       "  False))"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#数据池\n",
    "class Pool:\n",
    "\n",
    "    def __init__(self):\n",
    "        self.pool = []\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.pool)\n",
    "\n",
    "    def __getitem__(self, i):\n",
    "        return self.pool[i]\n",
    "\n",
    "    #更新动作池\n",
    "    def update(self):\n",
    "        #每次更新不少于N条新数据\n",
    "        old_len = len(self.pool)\n",
    "        while len(pool) - old_len < 200:\n",
    "            self.pool.extend(play()[0])\n",
    "\n",
    "        #只保留最新的N条数据\n",
    "        self.pool = self.pool[-2_0000:]\n",
    "\n",
    "    #获取一批数据样本\n",
    "    def sample(self):\n",
    "        data = random.sample(self.pool, 64)\n",
    "\n",
    "        state = torch.FloatTensor([i[0] for i in data]).reshape(-1, 4)\n",
    "        action = torch.LongTensor([i[1] for i in data]).reshape(-1, 1)\n",
    "        reward = torch.FloatTensor([i[2] for i in data]).reshape(-1, 1)\n",
    "        next_state = torch.FloatTensor([i[3] for i in data]).reshape(-1, 4)\n",
    "        over = torch.LongTensor([i[4] for i in data]).reshape(-1, 1)\n",
    "\n",
    "        return state, action, reward, next_state, over\n",
    "\n",
    "\n",
    "pool = Pool()\n",
    "pool.update()\n",
    "pool.sample()\n",
    "\n",
    "len(pool), pool[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "018c4b1e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0 445 -991.3\n",
      "100 20000 -906.0\n",
      "200 20000 200.0\n",
      "300 20000 200.0\n",
      "400 20000 200.0\n",
      "500 20000 141.35\n",
      "600 20000 200.0\n",
      "700 20000 200.0\n",
      "800 20000 200.0\n",
      "900 20000 200.0\n"
     ]
    }
   ],
   "source": [
    "#训练\n",
    "def train():\n",
    "    model.train()\n",
    "    optimizer = torch.optim.Adam(model.parameters(), lr=2e-4)\n",
    "    loss_fn = torch.nn.MSELoss()\n",
    "\n",
    "    #共更新N轮数据\n",
    "    for epoch in range(1000):\n",
    "        pool.update()\n",
    "\n",
    "        #每次更新数据后,训练N次\n",
    "        for i in range(200):\n",
    "\n",
    "            #采样N条数据\n",
    "            state, action, reward, next_state, over = pool.sample()\n",
    "\n",
    "            #计算value\n",
    "            value = model(state).gather(dim=1, index=action)\n",
    "\n",
    "            #计算target\n",
    "            with torch.no_grad():\n",
    "                target = model(next_state)\n",
    "            target = target.max(dim=1)[0].reshape(-1, 1)\n",
    "            target = target * 0.99 * (1 - over) + reward\n",
    "\n",
    "            loss = loss_fn(value, target)\n",
    "            loss.backward()\n",
    "            optimizer.step()\n",
    "            optimizer.zero_grad()\n",
    "\n",
    "        if epoch % 100 == 0:\n",
    "            test_result = sum([play()[-1] for _ in range(20)]) / 20\n",
    "            print(epoch, len(pool), test_result)\n",
    "\n",
    "\n",
    "train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "a101d0e6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjcuMSwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/bCgiHAAAACXBIWXMAAA9hAAAPYQGoP6dpAAATF0lEQVR4nO3df0xb57kH8K8N2EDgmBKKXS5Y4arVMm5+bCMJOcsfmxY3LEXTsnKlbYo6VkXpbWaipkyRhtRSpeoVVfpHt2wpuVLVpLpSxy7TzaaitBHXtERT3JDQIRGaoE3rLijNsZtmPjYk2ICf+0fHuXVDOgyOX5x8P9KR6vM+tp/zpuer4/NisImIgIhIAbvqBojo3sUAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZZQF0NGjR7FmzRoUFhaioaEBg4ODqlohIkWUBNBvfvMbtLW14bnnnsP777+PjRs3orGxEeFwWEU7RKSITcWXURsaGrB582b86le/AgAkk0nU1NRg//79+NnPfpbtdohIkfxsv2EikcDQ0BDa29utfXa7HT6fD8FgcMHnxONxxONx63EymcT169exevVq2Gy2O94zEaVHRBCLxVBVVQW7/fYftLIeQNeuXcPc3BzcbnfKfrfbjcuXLy/4nM7OThw6dCgb7RFRBk1MTKC6uvq241kPoKVob29HW1ub9dg0TXi9XkxMTEDTNIWdEdFCotEoampqUFpa+oV1WQ+giooK5OXlIRQKpewPhULweDwLPsfpdMLpdN6yX9M0BhDRCvaPbpFkfRXM4XCgvr4egUDA2pdMJhEIBKDrerbbISKFlHwEa2trQ0tLCzZt2oQtW7bg5z//OaampvD444+raIeIFFESQN///vfx8ccfo6OjA4Zh4Ctf+QrefvvtW25ME9HdTcnPAS1XNBqFy+WCaZq8B0S0Ai32HOV3wYhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKRM2gF05swZfOc730FVVRVsNht+97vfpYyLCDo6OvDAAw+gqKgIPp8Pf/rTn1Jqrl+/jt27d0PTNJSVlWHPnj2YnJxc1oEQUe5JO4CmpqawceNGHD16dMHxw4cP48iRIzh27BjOnTuHVatWobGxEdPT01bN7t27MTo6ir6+PvT29uLMmTN44oknln4URJSbZBkAyMmTJ63HyWRSPB6PvPTSS9a+SCQiTqdTfv3rX4uIyAcffCAA5Pz581bNW2+9JTabTa5cubKo9zVNUwCIaZrLaZ+I7pDFnqMZvQf04YcfwjAM+Hw+a5/L5UJDQwOCwSAAIBgMoqysDJs2bbJqfD4f7HY7zp07t+DrxuNxRKPRlI2Icl9GA8gwDACA2+1O2e92u60xwzBQWVmZMp6fn4/y8nKr5vM6OzvhcrmsraamJpNtE5EiObEK1t7eDtM0rW1iYkJ1S0SUARkNII/HAwAIhUIp+0OhkDXm8XgQDodTxmdnZ3H9+nWr5vOcTic0TUvZiCj3ZTSAamtr4fF4EAgErH3RaBTnzp2DrusAAF3XEYlEMDQ0ZNX09/cjmUyioaEhk+0Q0QqXn+4TJicn8ec//9l6/OGHH2J4eBjl5eXwer04cOAAXnjhBTz00EOora3Fs88+i6qqKuzatQsA8OUvfxnf/va3sXfvXhw7dgwzMzNobW3FD37wA1RVVWXswIgoB6S7vPbOO+8IgFu2lpYWEfl0Kf7ZZ58Vt9stTqdTtm/fLmNjYymv8cknn8gPf/hDKSkpEU3T5PHHH5dYLJbxJT4iUmOx56hNRERh/i1JNBqFy+WCaZq8H0S0Ai32HM2JVTAiujsxgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJl0v6zPETLNRufgjkxCvz97yHY7Ha4atYjz1GouDPKNgYQZV0idh1/6X8NkCQAwJ7vwL/8awcD6B7Ej2BEpAwDiIiUYQARkTIMICJShgFERMowgIhImbQCqLOzE5s3b0ZpaSkqKyuxa9cujI2NpdRMT0/D7/dj9erVKCkpQXNzM0KhUErN+Pg4mpqaUFxcjMrKShw8eBCzs7PLPxoiyilpBdDAwAD8fj/ee+899PX1YWZmBjt27MDU1JRV8/TTT+PNN99ET08PBgYG8NFHH+HRRx+1xufm5tDU1IREIoGzZ8/i9ddfx4kTJ9DR0ZG5oyKi3CDLEA6HBYAMDAyIiEgkEpGCggLp6emxai5duiQAJBgMiojIqVOnxG63i2EYVk1XV5domibxeHxR72uapgAQ0zSX0z4pMvXxuAz+x7/J4LG9Mnhsr1x41S83IyHVbVEGLfYcXdY9INM0AQDl5eUAgKGhIczMzMDn81k1a9euhdfrRTAYBAAEg0GsX78ebrfbqmlsbEQ0GsXo6OiC7xOPxxGNRlM2Isp9Sw6gZDKJAwcOYNu2bVi3bh0AwDAMOBwOlJWVpdS63W4YhmHVfDZ85sfnxxbS2dkJl8tlbTU1NUttm4hWkCUHkN/vx8WLF9Hd3Z3JfhbU3t4O0zStbWJi4o6/JxHdeUv6Mmprayt6e3tx5swZVFdXW/s9Hg8SiQQikUjKVVAoFILH47FqBgcHU15vfpVsvubznE4nnE7nUlolohUsrSsgEUFraytOnjyJ/v5+1NbWpozX19ejoKAAgUDA2jc2Nobx8XHoug4A0HUdIyMjCIfDVk1fXx80TUNdXd1yjoWIckxaV0B+vx9vvPEGfv/736O0tNS6Z+NyuVBUVASXy4U9e/agra0N5eXl0DQN+/fvh67r2Lp1KwBgx44dqKurw2OPPYbDhw/DMAw888wz8Pv9vMohusekFUBdXV0AgG9+85sp+48fP44f//jHAICXX34Zdrsdzc3NiMfjaGxsxCuvvGLV5uXlobe3F/v27YOu61i1ahVaWlrw/PPPL+9IiCjn2ET+/mvpckg0GoXL5YJpmtA0TXU7lKYb1yYw+t//fssvJCt0VSrujDJlsecovwtGRMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJSJq2/jEq0GCKCWCyGZDK54Hg8FlugPoppcSxYb7fbUVpaCpvNlvFeSS0GEGXc5OQkHn74YVy5cmXB8TXuUry0Zxvy7J9egMenp7Fz5yMw/nZjwfoHH3wQp0+fhtPpvGM9kxoMIMo4EYFhGLcNoGJbOSIz9+N/E/WwIYnq/EGEQiFc+Ti6YH1paSly8C+I0yKkdQ+oq6sLGzZsgKZp0DQNuq7jrbfessanp6fh9/uxevVqlJSUoLm5GaFQKOU1xsfH0dTUhOLiYlRWVuLgwYOYnZ3NzNFQTriZLMVwzIdwYg1CiX/GcGw74ski1W2RAmkFUHV1NV588UUMDQ3hwoUL+Na3voXvfve7GB0dBQA8/fTTePPNN9HT04OBgQF89NFHePTRR63nz83NoampCYlEAmfPnsXrr7+OEydOoKOjI7NHRSvabNKB6eQq6/HNuVLMSYHCjkgZWab77rtPXn31VYlEIlJQUCA9PT3W2KVLlwSABINBERE5deqU2O12MQzDqunq6hJN0yQejy/6PU3TFABimuZy26c7wDRN8Xq9AmDBrfafHpAjnf8lh144J4deOCdHXvhPeaCi/Lb1a9eulZs3b6o+LErDYs/RJd8DmpubQ09PD6ampqDrOoaGhjAzMwOfz2fVrF27Fl6vF8FgEFu3bkUwGMT69evhdrutmsbGRuzbtw+jo6P46le/mlYPly9fRklJyVIPge6QqakpzMzM3HY8Ev0bzgZeQnimFjaboCLvL5i8MXnb+ng8jsuXL8PhWHiVjFaeycnb/3t+VtoBNDIyAl3XMT09jZKSEpw8eRJ1dXUYHh6Gw+FAWVlZSr3b7YZhGAAAwzBSwmd+fH7sduLxOOLxuPU4Gv30ZqVpmrx/tALduHHjtkvwAPC32DS6/+c8gPOLer1kMgnTNFFQwI9puWJqampRdWkH0Je+9CUMDw/DNE389re/RUtLCwYGBtJuMB2dnZ04dOjQLfsbGhqgadodfW9KXzQazeiSeVFRERoaGlBYWJix16Q7a/4i4R9J+yehHQ4HHnzwQdTX16OzsxMbN27EL37xC3g8HiQSCUQikZT6UCgEj8cDAPB4PLesis0/nq9ZSHt7O0zTtLaJiYl02yaiFWjZX8VIJpOIx+Oor69HQUEBAoGANTY2Nobx8XHoug4A0HUdIyMjCIfDVk1fXx80TUNdXd1t38PpdFpL//MbEeW+tD6Ctbe3Y+fOnfB6vYjFYnjjjTfw7rvv4vTp03C5XNizZw/a2tpQXl4OTdOwf/9+6LqOrVu3AgB27NiBuro6PPbYYzh8+DAMw8AzzzwDv9/Pn3IlugelFUDhcBg/+tGPcPXqVbhcLmzYsAGnT5/Gww8/DAB4+eWXYbfb0dzcjHg8jsbGRrzyyivW8/Py8tDb24t9+/ZB13WsWrUKLS0teP755zN7VKSUzWZDaWlpxq5UudJ597KJ5N7PuEejUbhcLpimyY9jK1AymYRhGJibm8vI6+Xn58Pj8fDLqDlksecovwtGGWe321FVVaW6DcoB/H1ARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISJl81Q0shYgAAKLRqOJOiGgh8+fm/Ll6OzkZQJ988gkAoKamRnEnRPRFYrEYXC7XbcdzMoDKy8sBAOPj4194cJQqGo2ipqYGExMT0DRNdTs5gXO2NCKCWCyGqqqqL6zLyQCy2z+9deVyufg/xRJomsZ5SxPnLH2LuTjgTWgiUoYBRETK5GQAOZ1OPPfcc3A6napbySmct/Rxzu4sm/yjdTIiojskJ6+AiOjuwAAiImUYQESkDAOIiJTJyQA6evQo1qxZg8LCQjQ0NGBwcFB1S8p0dnZi8+bNKC0tRWVlJXbt2oWxsbGUmunpafj9fqxevRolJSVobm5GKBRKqRkfH0dTUxOKi4tRWVmJgwcPYnZ2NpuHosyLL74Im82GAwcOWPs4Z1kiOaa7u1scDoe89tprMjo6Knv37pWysjIJhUKqW1OisbFRjh8/LhcvXpTh4WF55JFHxOv1yuTkpFXz5JNPSk1NjQQCAblw4YJs3bpVvv71r1vjs7Ozsm7dOvH5fPLHP/5RTp06JRUVFdLe3q7ikLJqcHBQ1qxZIxs2bJCnnnrK2s85y46cC6AtW7aI3++3Hs/NzUlVVZV0dnYq7GrlCIfDAkAGBgZERCQSiUhBQYH09PRYNZcuXRIAEgwGRUTk1KlTYrfbxTAMq6arq0s0TZN4PJ7dA8iiWCwmDz30kPT19ck3vvENK4A4Z9mTUx/BEokEhoaG4PP5rH12ux0+nw/BYFBhZyuHaZoA/v8Lu0NDQ5iZmUmZs7Vr18Lr9VpzFgwGsX79erjdbqumsbER0WgUo6OjWew+u/x+P5qamlLmBuCcZVNOfRn12rVrmJubS/lHBwC3243Lly8r6mrlSCaTOHDgALZt24Z169YBAAzDgMPhQFlZWUqt2+2GYRhWzUJzOj92N+ru7sb777+P8+fP3zLGOcuenAog+mJ+vx8XL17EH/7wB9WtrGgTExN46qmn0NfXh8LCQtXt3NNy6iNYRUUF8vLyblmNCIVC8Hg8irpaGVpbW9Hb24t33nkH1dXV1n6Px4NEIoFIJJJS/9k583g8C87p/NjdZmhoCOFwGF/72teQn5+P/Px8DAwM4MiRI8jPz4fb7eacZUlOBZDD4UB9fT0CgYC1L5lMIhAIQNd1hZ2pIyJobW3FyZMn0d/fj9ra2pTx+vp6FBQUpMzZ2NgYxsfHrTnTdR0jIyMIh8NWTV9fHzRNQ11dXXYOJIu2b9+OkZERDA8PW9umTZuwe/du6785Z1mi+i54urq7u8XpdMqJEyfkgw8+kCeeeELKyspSViPuJfv27ROXyyXvvvuuXL161dpu3Lhh1Tz55JPi9Xqlv79fLly4ILqui67r1vj8kvKOHTtkeHhY3n77bbn//vvvqSXlz66CiXDOsiXnAkhE5Je//KV4vV5xOByyZcsWee+991S3pAyABbfjx49bNTdv3pSf/OQnct9990lxcbF873vfk6tXr6a8zl//+lfZuXOnFBUVSUVFhfz0pz+VmZmZLB+NOp8PIM5ZdvDXcRCRMjl1D4iI7i4MICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISJn/A5PZXCUmNc7ZAAAAAElFTkSuQmCC",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "200.0"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "play(True)[-1]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
